#!/bin/bash

BASE_DIR=`pwd`  #/home/neo/dns_hijack_data
ALEXA=$BASE_DIR"/alexa_top1m"
for DIR in $BASE_DIR $ALEXA 
do
    if [ ! -d $DIR ]
    then
        mkdir $DIR
    fi
done

TOP1M_ZIP="top-1m.csv.zip"
URL_ALEXA="http://s3.amazonaws.com/alexa-static/$TOP1M_ZIP"
TOP1M_DOWNLOAD="top-1m.csv"
TIMESTAMP=`date +%Y-%m-%d`

TOP1M_FILE=$ALEXA"/top-1m-processed-$TIMESTAMP.txt"
NEW_DOMAIN=$ALEXA"/new_domain_$TIMESTAMP.txt"
ALL_DOMAIN="${BASE_DIR}/all_domain_list.txt"

download_top1m(){
    echo "Downloading the top1m file, Creating $TOP1M_FILE"
    wget http://s3.amazonaws.com/alexa-static/$TOP1M_ZIP
}

process_top1m() {

    echo "Processing the top1m file, Creating $TOP1M_FILE"

    if [ ! -f $TOP1M_ZIP ]; then
        echo "$Top1M_ZIP does not exist!"
        exit 1
    fi

    unzip $TOP1M_ZIP
    cat $TOP1M_DOWNLOAD | cut -d "," -f2 | cut -d "/" -f1 | sort | uniq | awk '{printf "www.%s\n", $0}' > $TOP1M_FILE
    #cat $TOP1M_DOWNLOAD | cut -d "," -f2 | cut -d "/" -f1 | sort | uniq | awk '{printf "%s\n", $0}' > $TOP1M_FILE
    rm -f $TOP1M_ZIP
    rm -f $TOP1M_DOWNLOAD
}

update_domain() {
    echo "Creating $NEW_DOMAIN"
    cp $ALL_DOMAIN  temp.txt
    cat $TOP1M_FILE >> temp.txt
    cat temp.txt | sort | uniq -u > temp_new.txt
    cat $TOP1M_FILE >> temp_new.txt
    cat temp_new.txt | sort | uniq -d > $NEW_DOMAIN
    cp $NEW_DOMAIN new_domian.txt
    echo "Updating $ALL_DOMAIN"
    cat $TOP1M_FILE > temp.txt
    cat $ALL_DOMAIN >> temp.txt
    cat temp.txt | sort | uniq > $ALL_DOMAIN
}

#main()

download_top1m
process_top1m
update_domain


if [ -f temp.txt ]; then
rm -f temp.txt
fi

if [ -f temp_last.txt ]; then
rm -f temp_last.txt
fi

if [ -f temp_new.txt ]; then
rm -f temp_new.txt
fi

if [ -f temp_last_last.txt ]; then
rm -f temp_last_last.txt
fi
